#### This script will identify errors with batch processes on the HPC
#### It will read in the contents of the .Rout file generated with each process
#### And search for lines of text like 'Execution halted'
#### It can be used to search for specific errors, or in the case of errors associated with writing the .Rhistory file
#### Used to identify those specific errors and batch them out for processing again

#### First identify the directory where the .Rout and .sh files are located, and set it as the working directory

tmp.pbs = '/home1/99/jc152199/tmp.pbs/' ; setwd(tmp.pbs)

#### Now combine the list.files() function with the pattern call to identify the files of interest

tfiles = list.files(,pattern='.Rout')

#### Create a list to bind names of files that failed to execute, and didn't have proc.time() 

tfiles.exec.fail = NULL

count=0 #### Used to count 'execution halted' in loop

exec.fail.lengths= NULL  ##### Used to count the length of t.lines (as a proxy for the number of different types of failure that were experienced in the batch process)

fail.data = c(file.name = NA, length.fail = NA) ; fail.data = na.omit(fail.data)

#### Begin a loop to examine each file individually

for (tfile in tfiles) 
	
	{ 
	
	t.lines = readLines(tfile) #### Basic function for reading in all lines of the file named 'tfile' individually
	
	if(length(grep('Execution halted',t.lines))>0)    ### Here, the grep() function searches through t.lines for 'Execution halted', if the length of that is greater than zero then it will proceed with running commands in the curly brackets following the if statement
	
		{
		
		if(length(grep('proc.time()',t.lines))==0)
		
			{
		
			tname = gsub('.Rout','.sh',tfile) # Creates an element that is the name of .sh file where 'execution halted' and proc.time()==0, used to re-batch the fucked up files
			#count=count+1   #### Counts files
			#tfiles.exec.fail = rbind(tfiles.exec.fail,tfile)   #### Binds all failed files into a vector
			#exec.fail.lengths = rbind(exec.fail.lengths,length(t.lines))     #### Binds the length of all t.lines into a vector
			#fail.data = rbind(fail.data,cbind(tfile,length(t.lines)))
			
			system(paste('qsub -m n ',tname,sep=''))
			
			}
		
		} 

	}
	
# End Loop
	
n.unique.failures = unique(exec.fail.lengths)	# Unique list of length of tlines (the error message) when 'execution halted' occures and proc.time() does not

fail1 = fail.data[which(fail.data[,2]==23),] ; print(readLines(fail1[1,1]))
fail2 = fail.data[which(fail.data[,2]==24),] ; print(readLines(fail2[1,1]))
fail3 = fail.data[which(fail.data[,2]==26),] ; print(readLines(fail3[1]))

#Done

